from time import time
def print_metrics(estimator, X, y_true):
"""
Prints the precision, recall, and F1-score of a model.
Parameters
----------
estimator : sklearn estimator
The estimator to use for prediction.
X : numpy array
The data to predict on.
y_true : numpy array
The true labels for the data.
Returns
-------
None
"""
y_pred = estimator.predict(X)
precision, recall, f1score, _ = metrics.precision_recall_fscore_support(y_true, y_pred, average='binary')
print("Model metrics:")
print(" Precision:", precision)
print(" Recall:",recall)
print(" F1-score:",f1score)
def optimal_values(estimator, X, y_true, print_values=False, plot_charts=False):
"""
This function takes a model, X and y_true as input and returns the optimal threshold, f1score, precision and recall.
Parameters
----------
estimator : sklearn model
The model to be used for prediction.
X : numpy array
The input data.
y_true : numpy array
The true labels.
print_values : bool, optional
Whether to print the optimal values. The default is False.
plot_charts : bool, optional
Whether to plot the optimal confusion matrix and precision vs recall chart. The default is False.
Returns
-------
y_pred_proba : numpy array
The predicted probabilities for positive class.
threshold_opt : float
The optimal threshold.
f1score_opt : float
The optimal f1score.
precision_opt : float
The optimal precision score.
recall_opt : float
The optimal recall score.
thresholds : numpy array
The thresholds.
"""
y_pred_proba = estimator.predict_proba(X)[:,1] # predictions probability for positive class
precision, recall, thresholds = metrics.precision_recall_curve(y_true, y_pred_proba)
# # precision, recall manual calculations
# y = np.array(y_true).reshape((-1,1))
# pred = (y_pred_proba>= thresholds.reshape((-1,1))).T
# TP = np.sum(np.logical_and(y , pred ), axis=0)
# FN = np.sum(np.logical_and(y , np.logical_not(pred)), axis=0)
# FP = np.sum( np.logical_and(np.logical_not(y) , pred), axis=0)
# recall = np.append((TP /(TP+FN)), 0)
# precision = np.append((TP /(TP+FP)), 1)
# print("TP",TP, "FN",FN, "FP", FP,"recall", recall,"precision",precision)
f1score = 2*(precision*recall)/(precision+recall)
index = np.nanargmax(f1score)
threshold_opt = thresholds[index]
f1score_opt = f1score[index]
precision_opt = precision[index]
recall_opt = recall[index]
if print_values:
print("Optimal values:")
print(" threshold:", threshold_opt)
print(" precision:", precision_opt)
print(" recall:", recall_opt)
print(" F1-score:", f1score_opt)
if plot_charts:
fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(15,5))
y_pred = y_pred_proba >= threshold_opt
metrics.ConfusionMatrixDisplay.from_predictions(y_true, y_pred, ax=ax0)
sns.lineplot(x=range(len(precision)), y=precision, label="Precision", ax=ax1)
sns.lineplot(x=range(len(recall)), y=recall, label="Recall", ax=ax1)
ax= sns.lineplot(x=range(len(f1score)), y=f1score, label="f1 score", ax=ax1)
text_annot = f"TH {np.round(threshold_opt,3)}"
ax.annotate(text_annot, (index, 0), size=10)
text_annot = f"F1 {np.round(f1score_opt,2)}"
ax.annotate(text_annot, (index, f1score_opt), size=10)
xlim0, xlim1 = ax.get_xlim()
ylim0, ylim1= ax.get_ylim()
x_point = index/xlim1
y_point = f1score_opt/ylim1
ax.axhline(y=f1score_opt, xmax=x_point,linewidth=1, color='r', alpha=0.7,ls="--")
ax.axvline(x=index, ymax=y_point,linewidth=1, color='r', alpha=0.7,ls="--")
ax.set_xlabel("Threshold index")
ax.set_ylabel("Score")
# ax.set_xlim(-10,len(f1score)+10 )
minor_tick = np.arange(0, len(f1score), 100)
ax.set_xticks(minor_tick, minor=True)
ax.grid(visible=True, which='major', axis="x")
ax.grid(visible=True, which='minor',axis="x", alpha=0.4)
# ax.grid(which='minor', alpha=1)
for i in np.linspace(0, index, 6, dtype=int)[1:-1]:
recall_value = recall[i]
threshold_value = thresholds[i]
y_point = recall_value/ylim1
ax.axvline(x=i, ymax=y_point,linewidth=1, color='y', alpha=0.8,ls="--" )
text_annot = f"TH {np.round(threshold_value,3)}"
ax.annotate(text_annot, (i, recall_value-0.05), size=8)
plt.tight_layout()
return y_pred_proba, threshold_opt, f1score_opt, precision_opt, recall_opt, thresholds
def plot_recall_precision_threshold(y_true, y_pred_proba, threshold_index):
"""
This function takes y_true, y_pred_proba and a threshold_index as input, prints and make charts for precision, recall and f1 score with specific threshold.
Parameters
----------
y_true : numpy array
The true labels.
y_pred_proba : numpy array
The probabilities of predictions
threshold_index : int
Index of the threshold list
"""
precision, recall, thresholds = metrics.precision_recall_curve(y_true, y_pred_proba)
f1score = 2*(precision*recall)/(precision+recall)
index = threshold_index
threshold_opt = thresholds[index]
f1score_opt = f1score[index]
precision_opt = precision[index]
recall_opt = recall[index]
print("Optimal values:")
print(" threshold:", threshold_opt)
print(" precision:", precision_opt)
print(" recall:", recall_opt)
print(" F1-score:", f1score_opt)
fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(15,5))
y_pred = y_pred_proba >= threshold_opt
metrics.ConfusionMatrixDisplay.from_predictions(y_true, y_pred, ax=ax0)
sns.lineplot(x=range(len(precision)), y=precision, label="Precision", ax=ax1)
sns.lineplot(x=range(len(recall)), y=recall, label="Recall", ax=ax1)
ax= sns.lineplot(x=range(len(f1score)), y=f1score, label="f1 score", ax=ax1)
text_annot = f"TH {np.round(threshold_opt,3)}"
ax.annotate(text_annot, (index, 0), size=10)
text_annot = f"F1 {np.round(f1score_opt,2)}"
ax.annotate(text_annot, (index, f1score_opt), size=10)
text_annot = f"Recall {np.round(recall_opt,3)}"
ax.annotate(text_annot, (index, recall_opt), size=10)
xlim0, xlim1 = ax.get_xlim()
ylim0, ylim1= ax.get_ylim()
x_point = index/xlim1
y_point = recall_opt/ylim1
ax.axhline(y=f1score_opt, xmax=x_point,linewidth=1, color='r', alpha=0.7,ls="--")
ax.axhline(y=recall_opt, xmax=x_point,linewidth=1, color='r', alpha=0.7,ls="--")
ax.axvline(x=index, ymax=y_point,linewidth=1, color='r', alpha=0.7,ls="--")
ax.set_xlabel("Threshold index")
ax.set_ylabel("Score")
# ax.set_xlim(-10,len(f1score)+10 )
minor_tick = np.arange(0, len(f1score), 100)
ax.set_xticks(minor_tick, minor=True)
ax.grid(visible=True, which='major', axis="x")
ax.grid(visible=True, which='minor',axis="x", alpha=0.4)
plt.tight_layout()
